/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */

package Wrapper;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import crawler.UrlList;
import crawler.GetHTML;

/**
 *
 * @author Putra
 */
public class HTMLParser {
    private StringBuffer xmlText = new StringBuffer("<?xml version=\"1.0\"?>");
    private String baseUrl;
    private String xmlFileName;
    private String rootName;
    private String itemName;
    private int maxPage;
    private String htmlText;
    
    public HTMLParser(int maxPage){
        this.maxPage = maxPage;
    }
    
    private void retrievePages(int maxPage) {
        xmlText.append("\n" + "<" + rootName + ">");
        UrlList list = new UrlList(1,maxPage);
        for(int i=0;i<list.listUrl.size();i++){
            String pageUrl = list.listUrl.get(i);
            addElement(generateDivFieldRow(getHtmlText(pageUrl)));
            System.out.println("progress : "+(double)(i+1)/list.listUrl.size());
        }
        
        xmlText.append("\n" + "</" + rootName + ">");
    }
    
    public void addElement(List<ElementDiv> listDiv){
        
        StringBuffer pubXmlText = new StringBuffer("\n<" + itemName + ">\n");
        for(int i=0;i<listDiv.size();i++){
            String ruasElement = "<" + listDiv.get(i).getLabel() + "><![CDATA[" + listDiv.get(i).getValue() + "]]></"+listDiv.get(i).getLabel()+">";
            pubXmlText.append("  " + ruasElement + "\n");
        }
        pubXmlText.append("</" + itemName + ">");
        xmlText.append(pubXmlText);
    }
    
    public String getDiv(String component){
        String div = "div class=\"jrFieldRow #\">";
        div = div.replaceAll("#", component);
        
        return div;
    }
    
    private String getHtmlText(String targetUrl) {
        String htmlText = null;
        GetHTML getHTML= new GetHTML("",targetUrl);
        Thread thread = new Thread(getHTML);
        thread.run();
        htmlText = getHTML.getHtmlText();
        
        return htmlText;
    }
    
    private ElementDiv getTitle(String htmlText){
        ElementDiv div = new ElementDiv();
        div.setLabel("title");
        
        String startTitle = "<span itemprop=\"name\">";
        String endTitle = "</span>";
        int startPosTitle = htmlText.indexOf(startTitle);
        int endPosTitle = htmlText.indexOf(endTitle,startPosTitle+1);
        String title = "";
        if(startPosTitle!=-1 || endPosTitle!=-1)  title = htmlText.substring(startPosTitle+startTitle.length(), endPosTitle);
        div.setValue(cleanText(removeTags(title)));
        return div;
    }
    
    public List<ElementDiv> generateDivFieldRow(String htmlText){
        String regex = "\\<div class=\"jrFieldRow .*?\">";
        List<ElementDiv> elementDiv = new ArrayList<ElementDiv>();
        String[] itemList = new String[100];
        itemList = htmlText.split(regex);
        
        String forTitle = itemList[0];
        elementDiv.add(getTitle(forTitle));
        
        for(int i=1;i<itemList.length;i++){
            ElementDiv div = new ElementDiv();
            //System.out.println(item);
            String item = itemList[i];
            String startLabel = "<div class=\"jrFieldLabel\">";
            String endLabel = "</div>";
            int startPosLabel = item.indexOf(startLabel);
            int endPosLabel = item.indexOf(endLabel,startPosLabel+1);
            String label = item.substring(startPosLabel+startLabel.length(),endPosLabel);
            
            String startValue = "<div class=\"jrFieldValue \">";
            String endValue = "</div>";
            int startPosValue = item.indexOf(startValue);
            int endPosValue = item.indexOf(endValue,startPosValue+1);
            String value = item.substring(startPosValue+startValue.length(),endPosValue);
            
            div.setLabel(label);
            div.setValue(cleanText(removeTags(value)));
            elementDiv.add(div);
            
        }
        return elementDiv;
    }

    /**
     * @return the xmlText
     */
    public StringBuffer getXmlText() {
        return xmlText;
    }

    /**
     * @param xmlText the xmlText to set
     */
    public void setXmlText(StringBuffer xmlText) {
        this.xmlText = xmlText;
    }

    /**
     * @return the baseUrl
     */
    public String getBaseUrl() {
        return baseUrl;
    }

    /**
     * @param baseUrl the baseUrl to set
     */
    public void setBaseUrl(String baseUrl) {
        this.baseUrl = baseUrl;
    }

    /**
     * @return the xmlFileName
     */
    public String getXmlFileName() {
        return xmlFileName;
    }

    /**
     * @param xmlFileName the xmlFileName to set
     */
    public void setXmlFileName(String xmlFileName) {
        this.xmlFileName = xmlFileName;
    }

    /**
     * @return the rootName
     */
    public String getRootName() {
        return rootName;
    }

    /**
     * @param rootName the rootName to set
     */
    public void setRootName(String rootName) {
        this.rootName = rootName;
    }

    /**
     * @return the itemName
     */
    public String getItemName() {
        return itemName;
    }

    /**
     * @param itemName the itemName to set
     */
    public void setItemName(String itemName) {
        this.itemName = itemName;
    }

    /**
     * @return the htmlText
     */
    public String getHtmlText() {
        return htmlText;
    }

    /**
     * @param htmlText the htmlText to set
     */
    public void setHtmlText(String htmlText) {
        this.htmlText = htmlText;
    }

     /**
     * Return text without HTML tags
     * @param htmlText HTML text whose tags will be removed
     * @return Text without HTML tags
     */
    private String removeTags(String htmlText) {
        htmlText = htmlText.replaceAll("\\<.*?>", " ");
        htmlText = cleanText(htmlText);
        return htmlText;
    }

    private String cleanText(String text) {
        text = text.replaceAll("\\s+", " ");
        text = text.trim();
        return text;
    }
    

  
    
     private void saveToFile() {
        try {
            FileWriter file = new FileWriter(xmlFileName);
            BufferedWriter out = new BufferedWriter(file);
            out.write(xmlText.toString());
            out.close();
        } catch (IOException e) {
        }
    }
     
    
   public void start(){
       retrievePages(this.maxPage);
       saveToFile();
   }
}
